{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 建立基于用户的协同过滤算法模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle as pk\n",
    "import matplotlib.pyplot as plt\n",
    "#字典，用于建立用户和物品的索引\n",
    "from collections import defaultdict\n",
    "#稀疏矩阵，存储打分表\n",
    "import scipy.io as sio\n",
    "import scipy.sparse as ss\n",
    "#距离\n",
    "import scipy.spatial.distance as ssd\n",
    "import math\n",
    "import multiprocessing\n",
    "from multiprocessing import Pool\n",
    "from multiprocessing.managers import BaseManager\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = './Data/'\n",
    "model_path = './model/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 读入训练数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 用户和 item 的索引\n",
    "with open(model_path+'train_users_index.pkl', 'rb') as fr:\n",
    "    users_index = pk.load(fr)\n",
    "fr.close()\n",
    "\n",
    "with open(model_path+'train_items_index.pkl', 'rb') as fr:\n",
    "    items_index = pk.load(fr)\n",
    "fr.close()\n",
    "\n",
    "# 参与打分的用户和item数量\n",
    "user_num = len(users_index)\n",
    "item_num = len(items_index)\n",
    "\n",
    "# 用户和items 的倒排表\n",
    "with open(model_path+'user_items.pkl', 'rb') as fr:\n",
    "    user_items = pk.load(fr)\n",
    "fr.close()\n",
    "\n",
    "with open(model_path+'item_users.pkl', 'rb') as fr:\n",
    "    item_users = pk.load(fr)\n",
    "fr.close()\n",
    "\n",
    "# 用户物品关系矩阵\n",
    "user_item_scores = sio.mmread(model_path+'user_item_scores.mtx')\n",
    "user_item_scores = user_item_scores.tocsr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27764\n",
      "359810\n"
     ]
    }
   ],
   "source": [
    "user_num = len(users_index)\n",
    "item_num = len(items_index)\n",
    "print(user_num)\n",
    "print(item_num)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 计算训练数据每个用户的平均打分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 计算用户对所有打过分的物品的平均打分\n",
    "user_mean_score = np.zeros(user_num)\n",
    "# 遍历所有的用户\n",
    "for u in range(user_num):\n",
    "    user_score_sum = 0.0\n",
    "    items_num = 0\n",
    "    # 遍历用户打过分的items\n",
    "    for i in user_items[u]:\n",
    "        user_score_sum += user_item_scores[u,i]\n",
    "        items_num += 1\n",
    "    user_mean_score[u] = user_score_sum/items_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.54473439, 1.42251223, 1.47004608, 1.39130435, 1.        ,\n",
       "       1.28571429, 1.16315789, 1.74807692, 1.6185567 , 1.10526316,\n",
       "       1.06896552, 1.73015873, 1.11428571, 1.52941176, 1.75520833,\n",
       "       1.42009132, 1.71052632, 1.33333333, 1.11055276, 1.69855395,\n",
       "       1.25495751, 1.        , 1.15384615, 1.        , 1.2755102 ,\n",
       "       1.22049689, 1.86666667, 1.        , 1.04545455, 1.79676985,\n",
       "       1.11940299, 1.42307692, 1.78571429, 1.        , 1.4669967 ,\n",
       "       1.38679245, 1.24561404, 1.52941176, 1.41532258, 2.        ,\n",
       "       1.21186441, 1.57790368, 1.67326733, 1.32258065, 1.50954198,\n",
       "       1.32166667, 1.66666667, 1.71484375, 1.56603774, 1.53148615,\n",
       "       1.23684211, 1.17484663, 1.34169279, 1.48837209, 1.25183374,\n",
       "       1.94230769, 1.24210526, 1.2       , 1.56479058, 1.67920792,\n",
       "       1.29850746, 1.7082153 , 1.425     , 1.        , 1.39726027,\n",
       "       1.57575758, 1.14285714])"
      ]
     },
     "execution_count": 139,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_mean_score[28:95]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 计算两个用户之间的相似度"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "def user_similarity(uid1, uid2):\n",
    "    # uid1 和 uid2 共同打过分的items\n",
    "    common_items = {}\n",
    "    for item in user_items[uid1]:\n",
    "        if item in user_items[uid2]:\n",
    "            common_items[item] = 1\n",
    "    \n",
    "    # uid1 和 uid2 都打过分的item数为0， 相似度为 0\n",
    "    if len(common_items) == 0:\n",
    "        return 0\n",
    "    \n",
    "    # 计算 uid1 对common_items 的有效打分\n",
    "    uid1_valid_score = np.array([user_item_scores[uid1,item] - user_mean_score[uid1] for item in common_items])\n",
    "    # 计算 uid2 对common_items 的有效打分\n",
    "    uid2_valid_score = np.array([user_item_scores[uid2,item] - user_mean_score[uid2] for item in common_items])\n",
    "    \n",
    "    # 计算 uid1 和 uid2 的cosine相似度， ssd.cosine() 计算得到的是cosine距离\n",
    "    similarity = 1-ssd.cosine(uid1_valid_score, uid2_valid_score)\n",
    "    # uid1_valid_score 或者 uid2_valid_score 的模为 0，手动定义uid1 和 uid2 之间的相似度\n",
    "    if np.isnan(similarity):\n",
    "        similarity = 0.0\n",
    "    return similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.06415002990995822\n",
      "<class 'numpy.float64'>\n"
     ]
    }
   ],
   "source": [
    "temp_similarity = user_similarity(0, 10000)\n",
    "print(temp_similarity)\n",
    "print(type(temp_similarity))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "math.fabs(temp_similarity) < 0.3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 计算所有用户的相似性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # args: process_num 表示进程号, user_num 表示用户数， count_4_per_precess 每个进程需要处理的用户数量,user_similarity_matrix 用户相似度矩阵\n",
    "# def all_user_similarity(process_num,user_num,count_4_per_precess,user_similarity_matrix):\n",
    "#     if ((process_num+1)*count_4_per_precess) < user_num:\n",
    "#         for ui in range(process_num*count_4_per_precess,(process_num+1)*count_4_per_precess):\n",
    "#             if ui%50==0:\n",
    "#                 print('user_index i is: %d' % ui)\n",
    "#             user_similarity_matrix[ui,ui] = 1.0\n",
    "#             for uj in range(ui+1, user_num):\n",
    "#                 similarity = user_similarity(ui, uj)\n",
    "#                 if math.fabs(similarity) < 0.2:\n",
    "#                     continue\n",
    "#                 user_similarity_matrix[ui,uj] = similarity\n",
    "#                 user_similarity_matrix[uj,ui] = user_similarity_matrix[ui,uj]\n",
    "#     else:\n",
    "#         for ui in range(process_num*count_4_per_precess,user_num):\n",
    "#             if ui%50==0:\n",
    "#                 print('user_index i is: %d' % ui)\n",
    "#             user_similarity_matrix[ui,ui] = 1.0\n",
    "#             for uj in range(ui+1, user_num):\n",
    "#                 similarity = user_similarity(ui, uj)\n",
    "#                 if math.fabs(similarity) < 0.4:\n",
    "#                     continue\n",
    "#                 user_similarity_matrix[ui,uj] = similarity\n",
    "#                 user_similarity_matrix[uj,ui] = user_similarity_matrix[ui,uj]\n",
    "#     return user_similarity_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "# args: process_num 表示进程号, user_num 表示用户数， count_4_per_precess 每个进程需要处理的用户数量,user_similarity_matrix 用户相似度矩阵\n",
    "def all_user_similarity(process_num,user_num,count_4_per_precess,user_similarity_matrix_m):\n",
    "    if ((process_num+1)*count_4_per_precess) < user_num:\n",
    "        for ui in range(process_num*count_4_per_precess,(process_num+1)*count_4_per_precess):\n",
    "            if ui%100==0:\n",
    "                print('user_index i is: %d' % ui)\n",
    "            user_similarity_matrix_m.set_value(ui,ui,1.0)\n",
    "            for uj in range(ui+1, user_num):\n",
    "                similarity = user_similarity(ui, uj)\n",
    "                if math.fabs(similarity) < 0.2:\n",
    "                    continue\n",
    "                user_similarity_matrix_m.set_value(ui,uj,similarity)\n",
    "                user_similarity_matrix_m.set_value(uj,ui,similarity)\n",
    "    else:\n",
    "        for ui in range(process_num*count_4_per_precess,user_num):\n",
    "            if ui%100==0:\n",
    "                print('user_index i is: %d' % ui)\n",
    "            user_similarity_matrix_m.set_value(ui,ui,1.0)\n",
    "            for uj in range(ui+1, user_num):\n",
    "                similarity = user_similarity(ui, uj)\n",
    "                if math.fabs(similarity) < 0.2:\n",
    "                    continue\n",
    "                user_similarity_matrix_m.set_value(ui,uj,similarity)\n",
    "                user_similarity_matrix_m.set_value(uj,ui,similarity)\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24\n"
     ]
    }
   ],
   "source": [
    "print(multiprocessing.cpu_count()) # 机器的逻辑核数量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27764\n",
      "359810\n"
     ]
    }
   ],
   "source": [
    "user_num = len(users_index)\n",
    "item_num = len(items_index)\n",
    "print(user_num)\n",
    "print(item_num)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义一个要共享实例化对象的类\n",
    "class user_similarity_matrix_manager():\n",
    "    global user_num\n",
    "    def __init__(self):\n",
    "        print('user_num: ', user_num)\n",
    "        self.user_similarity_matrix = ss.dok_matrix((user_num, user_num), dtype=np.float64)\n",
    "\n",
    "    def set_value(self,ui,uj,value):\n",
    "        self.user_similarity_matrix[ui,uj] =value\n",
    "\n",
    "    def get_value(self,ui,uj):\n",
    "        return user_similarity_matrix[ui,uj]\n",
    "    \n",
    "    def get_all_value(self):\n",
    "        return self.user_similarity_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "def user_similarity_matrix_main(): \n",
    "    # 使用 BaseManager 实现多进程共享变量\n",
    "    manager = BaseManager()\n",
    "    # 一定要在start前注册，不然就注册无效\n",
    "    manager.register('user_similarity_matrix_manager', user_similarity_matrix_manager)\n",
    "    manager.start()\n",
    "    user_similarity_matrix_m = manager.user_similarity_matrix_manager()\n",
    "    # 进程数量 ()\n",
    "    multi_process_num = math.floor(multiprocessing.cpu_count()*0.85)\n",
    "    print('multi_process_num: %d' % multi_process_num)\n",
    "    p = Pool(multi_process_num)  \n",
    "    count_4_per_precess = math.ceil(user_num / multi_process_num)   # 每个进程需要处理的人数\n",
    "    for process_num in range(multi_process_num):\n",
    "        p.apply_async(all_user_similarity,args=(process_num,user_num,count_4_per_precess,user_similarity_matrix_m))\n",
    "    p.close()\n",
    "    p.join()\n",
    "    print('all subprocessed done')\n",
    "    return user_similarity_matrix_m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "user_num:  100\n",
      "multi_process_num: 20\n",
      "user_index i is: 0\n",
      "all subprocessed done\n"
     ]
    }
   ],
   "source": [
    "user_similarity_matrix_m = user_similarity_matrix_main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (10, 10)\t1.0\n",
      "  (60, 60)\t1.0\n",
      "  (20, 20)\t1.0\n",
      "  (75, 75)\t1.0\n",
      "  (30, 30)\t1.0\n",
      "  (25, 25)\t1.0\n",
      "  (15, 15)\t1.0\n",
      "  (5, 5)\t1.0\n",
      "  (45, 45)\t1.0\n",
      "  (85, 85)\t1.0\n",
      "  (60, 62)\t0.4547035447597877\n",
      "  (50, 50)\t1.0\n",
      "  (15, 16)\t1.0\n",
      "  (62, 60)\t0.4547035447597877\n",
      "  (20, 22)\t0.496138938356834\n",
      "  (75, 77)\t0.3071188611862593\n",
      "  (85, 87)\t0.2449751827538179\n",
      "  (45, 46)\t-1.0\n",
      "  (22, 20)\t0.496138938356834\n",
      "  (70, 70)\t1.0\n",
      "  (60, 63)\t-0.6923984912250534\n",
      "  (6, 6)\t1.0\n",
      "  (87, 85)\t0.2449751827538179\n",
      "  (20, 23)\t1.0\n",
      "  (6, 10)\t-0.2533783278079953\n",
      "  :\t:\n",
      "  (74, 29)\t-1.0\n",
      "  (29, 76)\t0.2406566296126701\n",
      "  (76, 29)\t0.2406566296126701\n",
      "  (29, 78)\t-0.24595119499656315\n",
      "  (78, 29)\t-0.24595119499656315\n",
      "  (29, 81)\t0.3794970572541905\n",
      "  (81, 29)\t0.3794970572541905\n",
      "  (29, 83)\t1.0\n",
      "  (83, 29)\t1.0\n",
      "  (29, 88)\t0.4144886454587544\n",
      "  (88, 29)\t0.4144886454587544\n",
      "  (29, 90)\t-0.6674890357138656\n",
      "  (90, 29)\t-0.6674890357138656\n",
      "  (29, 92)\t0.4016246741445685\n",
      "  (92, 29)\t0.4016246741445685\n",
      "  (29, 93)\t0.4471999433782402\n",
      "  (93, 29)\t0.4471999433782402\n",
      "  (29, 94)\t1.0\n",
      "  (94, 29)\t1.0\n",
      "  (29, 95)\t0.3676381387267684\n",
      "  (95, 29)\t0.3676381387267684\n",
      "  (29, 97)\t0.2516089396890231\n",
      "  (97, 29)\t0.2516089396890231\n",
      "  (29, 99)\t-0.2279272341419094\n",
      "  (99, 29)\t-0.2279272341419094\n"
     ]
    }
   ],
   "source": [
    "user_similarity_matrix = user_similarity_matrix_m.get_all_value()\n",
    "print(user_similarity_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(model_path+'user_similarity_matrix.pkl', 'wb') as fw:\n",
    "    pk.dump(user_similarity_matrix, fw)\n",
    "fw.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'scipy.sparse.dok.dok_matrix'>\n"
     ]
    }
   ],
   "source": [
    "print(type(user_similarity_matrix))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(model_path+'user_similarity_matrix.pkl', 'rb') as fr:\n",
    "#     user_similarity_matrix = pk.load(fr)\n",
    "# fr.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(user_similarity_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
